package com.yd.tool;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
/*
 * 通过统计网址获取地区数据 http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/
 * @author shangruihua
 *
 */
public class JsoupCitysURLDB {
	public static void main(String[] args) throws IOException {
		
		SimpleDateFormat df_ks = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		String kaishiTime=df_ks.format(new Date());
        System.out.println("开始----------------"+kaishiTime);
		
		//省
		List<Map<String, String>> listsheng=new ArrayList<Map<String,String>>();
		File sheng = new File("/Users/shangruihua/Downloads/jsoup/citys/citys_shengDB.txt");
		
		String urlruls="http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/index.html";
		Document doc = Jsoup.parse(new URL(urlruls).openStream(), "GBK", urlruls);
//		Document doc = Jsoup.connect("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/index.html").header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0")
//				.header("Connection", "close")//如果是这种方式，这里务必带上
//				.maxBodySize(9999999)
//				.timeout(8000)
//				.get(); 
		Elements times_content = doc.getElementsByClass("provincetr").select("a");
		System.out.println("省市的总数:"+times_content.size());//---省市的总数31------------一个一个弄
			
		for (int i = 0; i < times_content.size(); i++) {
			
			String url=times_content.get(i).attr("href");//---------------一个一个弄
			String id=url.replace(".html", "")+"0000000000";
			String title=times_content.get(i).text().replace(Jsoup.parse("&nbsp;").text(),""); 
			String tid="0";
			String pid="0";
			String ison="0";
			String shouzimu=UtsHanziToPinyin.getFirstLetter(title);
			String quanpin=UtsHanziToPinyin.toHanyuPinyin(title);
			
			
			Map<String, String> m=new HashMap<String, String>();
			m.put("url", url);
			m.put("id", id);
			listsheng.add(m);
			intxt(sheng,"insert into citys (idstr,title,tid,pid,ison,shouzimu,quanpin) values ('"+id+"','"+ title+ "','"+tid+"','"+pid+"','"+ison+"','"+ shouzimu+ "','"+ quanpin + "');");
			System.out.println(title+"__"+url);
		}
			
		
		
		try{Thread.sleep(5000);/*线程睡眠1秒！\n*/
		System.out.println("---------省完成睡眠5秒---------");
		}catch (InterruptedException e) {e.printStackTrace();}
		
		
		//市
		List<Map<String, String>> listshi=new ArrayList<Map<String,String>>();
		File shi = new File("/Users/shangruihua/Downloads/jsoup/citys/citys_shiDB.txt");
		for (int i = 0; i < listsheng.size(); i++) {
			Map<String, String> listm=listsheng.get(i);
			
			String urlrul="http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/"+listm.get("url");
			Document docc = Jsoup.parse(new URL(urlrul).openStream(), "GBK", urlrul);
			
//			Document docc = Jsoup.connect("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/"+listm.get("url")).header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0")
//					.header("Connection", "close")//如果是这种方式，这里务必带上
//					.maxBodySize(9999999)
//					.timeout(8000)//超时时间
//					.get();
			Elements times_contentt = docc.getElementsByClass("citytr").select("a");
			String url="";
			String id="";
			String title="";
			String tid="1";
			String pid=listm.get("id");
			String ison="0";
			String shouzimu="";
			String quanpin="";
			
			for (int j = 0; j < times_contentt.size(); j++) {
				if(j%2==0){
					id=times_contentt.get(j).text().replace(Jsoup.parse("&nbsp;").text(),"");
				}else{
					url=times_contentt.get(j).attr("href");
					title=times_contentt.get(j).text();
					shouzimu=UtsHanziToPinyin.getFirstLetter(title);
					quanpin=UtsHanziToPinyin.toHanyuPinyin(title);
					
					
					Map<String, String> m=new HashMap<String, String>();
					m.put("url", url);
					m.put("id", id);
					listshi.add(m);
					
					intxt(shi,"insert into citys (idstr,title,tid,pid,ison,shouzimu,quanpin) values ('"+id+"','"+ title+ "','"+tid+"','"+pid+"','"+ison+"','"+ shouzimu+ "','"+ quanpin + "');");
//					System.out.println(id+"___"+title+"___"+pid+"__"+url+"__"+shouzimu+"__"+quanpin);
					System.out.println(title);
				}
				
			}
			
		}
		
		
		try{Thread.sleep(5000);/*线程睡眠1秒！\n*/
		System.out.println("---------市完成睡眠5秒---------");
		}catch (InterruptedException e) {e.printStackTrace();}
		
		//区县
		List<Map<String, String>> listqu=new ArrayList<Map<String,String>>();
		File qu = new File("/Users/shangruihua/Downloads/jsoup/citys/citys_quDB.txt");
		for (int i = 0; i < listshi.size(); i++) {
			Map<String, String> listm=listshi.get(i);
			System.out.println("==================="+listm.get("url")+"===================");
			
			String urlrul="http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/"+listm.get("url");
			Document docc = Jsoup.parse(new URL(urlrul).openStream(), "GBK", urlrul);
//			Document docc = Jsoup.connect("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/"+listm.get("url")).header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0")
//					.header("Connection", "close")//如果是这种方式，这里务必带上
//					.maxBodySize(9999999)
//					.timeout(8000)//超时时间
//					.get();
			Elements times_contentt = docc.getElementsByClass("countytr").select("a");
			String url="";
			String id="";
			String title="";
			String tid="2";
			String pid=listm.get("id");
			String ison="0";
			String shouzimu="";
			String quanpin="";
			
			for (int j = 0; j < times_contentt.size(); j++) {
				if(j%2==0){
					id=times_contentt.get(j).text().replace(Jsoup.parse("&nbsp;").text(),"");
				}else{
					url=id.substring(0,2)+"/"+times_contentt.get(j).attr("href");
					title=times_contentt.get(j).text().replace(Jsoup.parse("&nbsp;").text(),"");
					shouzimu=UtsHanziToPinyin.getFirstLetter(title);
					quanpin=UtsHanziToPinyin.toHanyuPinyin(title);
					
					Map<String, String> m=new HashMap<String, String>();
					m.put("url", url);
					m.put("id", id);
					listqu.add(m);
					
					intxt(qu,"insert into citys (idstr,title,tid,pid,ison,shouzimu,quanpin) values ('"+id+"','"+ title+ "','"+tid+"','"+pid+"','"+ison+"','"+ shouzimu+ "','"+ quanpin + "');");
					//System.out.println(id+"___"+title+"___"+pid+"__"+url+"__"+shouzimu+"__"+quanpin);
					System.out.println(id+"__"+title+"__"+url);
				}
				
			}
		
		
		}
		
		

		
		
		try{Thread.sleep(5000);/*线程睡眠1秒！\n*/
		System.out.println("---------区完成睡眠5秒---------");
		}catch (InterruptedException e) {e.printStackTrace();}
		
		
		//路
		List<Map<String, String>> listlu=new ArrayList<Map<String,String>>();
		File lu = new File("/Users/shangruihua/Downloads/jsoup/citys/citys_luDB.txt");
		for (int i = 0; i < listqu.size(); i++) {
			
			try{Thread.sleep(50);/*线程睡眠1秒！\n*/
			System.out.println("---------路每个开始睡眠5秒---------");
			}catch (InterruptedException e) {e.printStackTrace();}
			
			
			Map<String, String> listm=listqu.get(i);
			System.out.println(listm.get("url"));
			
			String urlrul="http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/"+listm.get("url");
			Document docc = Jsoup.parse(new URL(urlrul).openStream(), "GBK", urlrul);
			
//			Document docc = Jsoup.connect("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2016/"+listm.get("url")).header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0")
//					.header("Connection", "close")//如果是这种方式，这里务必带上
//					.maxBodySize(9999999)
//					.timeout(8000)//超时时间
//					.get();
			Elements times_contentt = docc.getElementsByClass("towntr").select("a");
			String url="";
			String id="";
			String title="";
			String tid="3";
			String pid=listm.get("id");
			String ison="0";
			String shouzimu="";
			String quanpin="";
			
			for (int j = 0; j < times_contentt.size(); j++) {
				if(j%2==0){
					id=times_contentt.get(j).text().replace(Jsoup.parse("&nbsp;").text(),"");
				}else{
					url=times_contentt.get(j).attr("href");
					title=times_contentt.get(j).text().replace(Jsoup.parse("&nbsp;").text(),"").replace("生态文化旅游示范区管理委员会", "");
					if(!title.equals("街道办事处")){
						title=title.replace("街道", "").replace("办事处", "").replace("生态文化旅游示范区管理委员会", "");
					}
					shouzimu=UtsHanziToPinyin.getFirstLetter(title);
					quanpin=UtsHanziToPinyin.toHanyuPinyin(title);
					intxt(lu,"insert into citys (idstr,title,tid,pid,ison,shouzimu,quanpin) values ('"+id+"','"+ title+ "','"+tid+"','"+pid+"','"+ison+"','"+ shouzimu+ "','"+ quanpin + "');");
					System.out.println(id+"___"+title+"___"+pid+"__"+url+"__"+shouzimu+"__"+quanpin);
					
					//Map<String, String> m=new HashMap<String, String>();
					//m.put("url", url);
					//m.put("id", id);
					//listlu.add(m);
				}
				
			}
		
		
		}
		
		System.out.println();
		System.out.println();
		System.out.println();
		System.out.println("----end----");
		
		SimpleDateFormat df_js = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		String jieshuTime=df_js.format(new Date());
        System.out.println("结束--------"+kaishiTime+"--------"+jieshuTime);
		
		
		
	}
	
	
	
	
	
	
	//写入txt 追加
	public static void intxt(File file,String conent) {
		BufferedWriter out = null;
		try {
			out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true)));
			out.write(conent+"\r\n");
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			try {
				out.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
}
